package main;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;

import common.Normalizer;
import common.Polarity;

public class Parse 
{
	private HashMap<String,ArrayList<String>> irrMsgs;
	private HashMap<String,ArrayList<String>> neuMsgs;
	private HashMap<String,ArrayList<String>> posMsgs;
	private HashMap<String,ArrayList<String>> negMsgs;
	
	private HashMap<String, HashMap<String, Integer>> irrWordsCountIndex;
	private HashMap<String, HashMap<String, Integer>> neuWordsCountIndex;
	private HashMap<String, HashMap<String, Integer>> posWordsCountIndex;
	private HashMap<String, HashMap<String, Integer>> negWordsCountIndex;
	
	//Not used
	private ArrayList<String> irrTop20Frequency;
	private ArrayList<String> neuTop20Frequency;
	private ArrayList<String> posTop20Frequency;
	private ArrayList<String> negTop20Frequency;
	
	private HashMap<String, HashMap<String,Double>> irrBetas;
	private HashMap<String, HashMap<String,Double>> neuBetas;
	private HashMap<String, HashMap<String,Double>> posBetas;
	private HashMap<String, HashMap<String,Double>> negBetas;
	
	private int irrTotalWords;
	private int neuTotalWords;
	private int posTotalWords;
	private int negTotalWords;
	
	//Not used
	private int alltotalWords;
	
	//Not used
	private int irrWordsDiff;
	private int neuWordsDiff;
	private int posWordsDiff;
	private int negWordsDiff;
	
	//Not used
	private int irrAverageLength;
	private int neuAverageLength;
	private int posAverageLength;
	private int negAverageLength;
	
	public Parse()
	{
		irrMsgs = new HashMap<String,ArrayList<String>>();
		neuMsgs = new HashMap<String,ArrayList<String>>();
		posMsgs = new HashMap<String,ArrayList<String>>();
		negMsgs = new HashMap<String,ArrayList<String>>();

		irrWordsCountIndex = new HashMap<String,HashMap<String,Integer>>();
		neuWordsCountIndex = new HashMap<String,HashMap<String,Integer>>();
		posWordsCountIndex = new HashMap<String,HashMap<String,Integer>>();
		negWordsCountIndex = new HashMap<String,HashMap<String,Integer>>();
		
		irrTop20Frequency = new ArrayList<String>();
		neuTop20Frequency = new ArrayList<String>();
		posTop20Frequency = new ArrayList<String>();
		negTop20Frequency = new ArrayList<String>();
		
		irrBetas = new HashMap<String, HashMap<String,Double>>();
		neuBetas = new HashMap<String, HashMap<String,Double>>();
		posBetas = new HashMap<String, HashMap<String,Double>>();
		negBetas = new HashMap<String, HashMap<String,Double>>();
		
		try 
		{
			Normalizer.Init();
		} 
		catch (IOException e) 
		{
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}
	
	//Read "train.txt" and classify the messages according to their categories and marques
	public void ClassifyMsgs(String fileName) throws IOException
	{
		BufferedReader reader = new BufferedReader(new FileReader(new File(fileName)));
		String line = null;
		while((line = reader.readLine()) != null)
		{
			line = line.toLowerCase();
			int rPara = line.indexOf(')');
			
			String head = line.substring(0,rPara+1);
			String content = line.substring(rPara+1);
			
			String marque = GetMarque(head);
			if(null == marque)
			{
				System.out.println("Ilegal marque!");
				System.exit(0);
			}
			
			if(head.contains("irrelevant"))
			{
				InsertMsg(irrMsgs,content,marque);
			}
			else if(head.contains("neutral"))
			{
				InsertMsg(neuMsgs,content,marque);
			}
			else if(head.contains("positive"))
			{
				InsertMsg(posMsgs,content,marque);
			}
			else if(head.contains("negative"))
			{
				InsertMsg(negMsgs,content,marque);
			}
			else
			{
				System.out.println("Ilegal content!");
				System.exit(0);
			}
		}
	}

	//Calculate statistical informations
	public void Statistic() throws IOException
	{
		ConstructWordsCountIndex(irrMsgs,irrWordsCountIndex);
		ConstructWordsCountIndex(neuMsgs,neuWordsCountIndex);
		ConstructWordsCountIndex(posMsgs,posWordsCountIndex);
		ConstructWordsCountIndex(negMsgs,negWordsCountIndex);

		GetTop20Frequency(irrWordsCountIndex,irrTop20Frequency);
		GetTop20Frequency(neuWordsCountIndex,neuTop20Frequency);
		GetTop20Frequency(posWordsCountIndex,posTop20Frequency);
		GetTop20Frequency(negWordsCountIndex,negTop20Frequency);
		
		//number of different words for each category
		irrWordsDiff = irrWordsCountIndex.size();
		neuWordsDiff = neuWordsCountIndex.size();
		posWordsDiff = posWordsCountIndex.size();
		negWordsDiff = negWordsCountIndex.size();
		
		//Average length of messages for each category
		irrAverageLength = GetAverageMsgLength(irrMsgs);
		neuAverageLength = GetAverageMsgLength(neuMsgs);
		posAverageLength = GetAverageMsgLength(posMsgs);
		negAverageLength = GetAverageMsgLength(negMsgs);
		
		//Calculate the total words count for each category
		irrTotalWords = GetTotalWordsCount(irrWordsCountIndex);
		neuTotalWords = GetTotalWordsCount(neuWordsCountIndex);
		posTotalWords = GetTotalWordsCount(posWordsCountIndex);
		negTotalWords = GetTotalWordsCount(negWordsCountIndex);
		alltotalWords = irrTotalWords + neuTotalWords + posTotalWords + negTotalWords;
		
		//parameter betas
		CalculateBetas(irrWordsCountIndex, irrTotalWords, irrBetas);
		CalculateBetas(neuWordsCountIndex, neuTotalWords, neuBetas);
		CalculateBetas(posWordsCountIndex, posTotalWords, posBetas);
		CalculateBetas(negWordsCountIndex, negTotalWords, negBetas);
	}
	
	//Get the beta value of word of a marque
	public double GetBeta(Polarity p, String word, String marque)
	{
		word = word.toLowerCase();
		switch(p)
		{
		case Irrelevant:
			if(irrBetas.containsKey(word))
			{
				if(irrBetas.get(word).containsKey(marque))
				{
					double b = irrBetas.get(word).get(marque);
					return b;
				}
			}
			return 0;
			
		case Neutral:
			if(neuBetas.containsKey(word))
			{
				if(neuBetas.get(word).containsKey(marque))
				{
					double b = neuBetas.get(word).get(marque);
					return b;
				}
			}
			return 0;
			
		case Positive:
			if(posBetas.containsKey(word))
			{
				if(posBetas.get(word).containsKey(marque))
				{
					double b = posBetas.get(word).get(marque);
					return b;
				}
			}
			return 0;
			
		case Negative:
			if(negBetas.containsKey(word))
			{
				if(negBetas.get(word).containsKey(marque))
				{
					double b = negBetas.get(word).get(marque);
					return b;
				}
			}
			return 0;
			
		default:
			return 0;
		}
	}
	
	//Calculate the average message length for each category
	private int GetAverageMsgLength(HashMap<String,ArrayList<String>> msgs) throws IOException
	{
		int len = 0;
		int msgCount = 0;
		Set<String> keys = msgs.keySet();
		for(Iterator<String> itr = keys.iterator(); itr.hasNext(); )
		{
			String marque = itr.next();
			ArrayList<String> ms = msgs.get(marque);
			for(int i=0; i<ms.size(); ++i)
			{
				ArrayList<String> words = Normalizer.GetWords(ms.get(i));
				len += words.size();
				++msgCount;
			}
		}
		len /= msgCount;
		
		return len;
	}
	
	private int GetTotalWordsCount(Map<String, HashMap<String,Integer>> wordsCountIndex)
	{
		int totalCount = 0;
		
		Set<String> keys = wordsCountIndex.keySet();
		for(Iterator<String> itr = keys.iterator(); itr.hasNext(); )
		{
			String word = itr.next();
			HashMap<String,Integer> count = wordsCountIndex.get(word);
			Set<String> marques = count.keySet();
			for(Iterator<String> m = marques.iterator(); m.hasNext(); )
			{
				totalCount += count.get(m.next());
			}
		}
		System.out.println(totalCount);
		return totalCount;
	}
	
	//Calculate the top 20 frequent words in "words", put the result into "top20"
	private void GetTop20Frequency(HashMap<String,HashMap<String,Integer>> words, ArrayList<String> top20)
	{
		//Get the list of entries of all the words
		ArrayList<Map.Entry<String, HashMap<String,Integer>>> entries = new ArrayList<Map.Entry<String, HashMap<String,Integer>>>(words.entrySet());
		//Sort the entires according their frequency
		Collections.sort(entries, new Comparator<Map.Entry<String, HashMap<String,Integer>>>()
				{
					public int compare(Map.Entry<String, HashMap<String,Integer>> c1, Map.Entry<String, HashMap<String,Integer>> c2)
					{
						Map<String,Integer> tmp1 = c1.getValue();
						Map<String,Integer> tmp2 = c2.getValue();
						int sum1 = 0, sum2 = 0;
						Set<String> keys1 = tmp1.keySet();
						for(Iterator<String> itr = keys1.iterator(); itr.hasNext(); )
						{
							sum1 += tmp1.get(itr.next());
						}
						Set<String> keys2 = tmp2.keySet();
						for(Iterator<String> itr = keys2.iterator(); itr.hasNext(); )
						{
							sum2 += tmp2.get(itr.next());
						}
						return sum2 - sum1;
					}
				}
		);
		
		//Get the top 20
		int size = Math.min(20, entries.size());
		for(int i=0; i<size; ++i)
		{
			top20.add(entries.get(i).getKey());
			Map<String,Integer> tmp = entries.get(i).getValue();
			/*
			Set<String> ms = tmp.keySet();
			int c = 0;
			for(Iterator<String> itr=ms.iterator(); itr.hasNext(); )
			{
				c += tmp.get(itr.next());
			}
			//	System.out.println("	"+entries.get(i).getKey()+"	"+c);
			*/
		}
	}
	
	//Calculate the count of each word for each category
	/*
	 * 	Word1:  wordsNum(Microsoft)	wordsNum(Google)  wordsNum(Apple)  wordsNum(Twitter)
	 * 	Word2:  wordsNum(Microsoft)	wordsNum(Google)  wordsNum(Apple)  wordsNum(Twitter)
	 * 	Word3:  wordsNum(Microsoft)	wordsNum(Google)  wordsNum(Apple)  wordsNum(Twitter)
	 * 	...
	 * 	...
	 */
	private void ConstructWordsCountIndex(HashMap<String,ArrayList<String>> contents, HashMap<String,HashMap<String,Integer>> map) throws IOException
	{
		Set<String> keys = contents.keySet();
		//Iterate for 4 marques
		for(Iterator<String> itr = keys.iterator(); itr.hasNext(); )
		{
			//Marque name
			String marque = itr.next();
			//All the messages for this marque
			ArrayList<String> msgs = contents.get(marque);
			for(int i=0; i<msgs.size(); ++i)
			{
				ArrayList<String> words = Normalizer.GetWords(msgs.get(i));
				for(String word: words)
				{
					word = word.toLowerCase();
					if(map.containsKey(word))
					{
						HashMap<String,Integer> count = map.get(word);
						if(count.containsKey(marque))
						{
							int n = count.get(marque);
							count.put(marque, ++n);
						}
						else
						{
							count.put(marque, 1);
						}
						map.put(word, count);
					}
					else
					{
						HashMap<String,Integer> count = new HashMap<String,Integer>();
						count.put(marque, 1);
						map.put(word, count);
					}
				}
			}
		}
	}
	
	//Calculate the parameters 'beta' for each category
	//For each category, there are a sequence of betas for each marque
	// Beta = wordNum(marque1) / totalWordsOfTheCategory
	/*
	 *	Word1: Beta(Microsoft)	Beta(Google)	Beta(Apple)	Beta(Twitter)
	 *	Word2:
	 *	Word3:
	 *	...
	 *	...
	 */
	private void CalculateBetas(Map<String, HashMap<String,Integer>> wordsCountIndex, 
			int totalWords, 
			Map<String, HashMap<String,Double>> wordsBeta)
	{
		Set<String> keys = wordsCountIndex.keySet();
		for(Iterator<String> itr = keys.iterator(); itr.hasNext(); )
		{
			String word = itr.next();
			HashMap<String, Double> betas = new HashMap<String,Double>();
			
			HashMap<String, Integer> counts = wordsCountIndex.get(word);
			Set<String> marques = counts.keySet();
			for(Iterator<String> m = marques.iterator(); m.hasNext(); )
			{
				String marque = m.next();
				int count = counts.get(marque);
				// For each category: beta = wordNum / totalWords;
				betas.put(marque, 1.0*count/totalWords);
			}
			
			wordsBeta.put(word, betas);
		}
		
	}
	
	//Get the name of the marque according to the head of each message
	private String GetMarque(String head)
	{
		String marque = null;
		if(head.contains("microsoft"))
		{
			marque = "microsoft";
		}
		else if(head.contains("apple"))
		{
			marque = "apple";
		}
		else if(head.contains("google"))
		{
			marque = "google";
		}
		else if(head.contains("twitter"))
		{
			marque = "twitter";
		}
		
		return marque;
	}
	
	//Put the message of of certain marque into the map
	/*
	 * 	Microsoft: msg1, msg2, msg3, ...
	 * 	Google:	   msg1, msg2, msg3, ...
	 * 	Apple:	   msg1, msg2, msg3, ...
	 * 	Twitter:   msg1, msg2, msg3, ...
	 */
	private void InsertMsg(Map<String,ArrayList<String>> msgs, String m, String marque)
	{
		if(msgs.containsKey(marque))
		{
			ArrayList<String> ms = msgs.get(marque);
			ms.add(m);
			msgs.put(marque, ms);
		}
		else
		{
			ArrayList<String> ms = new ArrayList<String>();
			ms.add(m);
			msgs.put(marque, ms);
		}
	}
	
	public void ShowBeta(Polarity p)
	{
		if(p == Polarity.Irrelevant)
		{
			Set<String> words = irrBetas.keySet();
			for(Iterator<String> itr = words.iterator(); itr.hasNext(); )
			{
				String word = itr.next();
				System.out.println(word+"\t"+irrBetas.get(word));
			}
		}
		else if(p == Polarity.Neutral)
		{
			Set<String> words = neuBetas.keySet();
			for(Iterator<String> itr = words.iterator(); itr.hasNext(); )
			{
				String word = itr.next();
				System.out.println(word+"\t"+neuBetas.get(word));
			}
		}
		else if(p == Polarity.Positive)
		{
			Set<String> words = posBetas.keySet();
			for(Iterator<String> itr = words.iterator(); itr.hasNext(); )
			{
				String word = itr.next();
				System.out.println(word+"\t"+posBetas.get(word));
			}
		}
		else if(p == Polarity.Negative)
		{
			Set<String> words = negBetas.keySet();
			for(Iterator<String> itr = words.iterator(); itr.hasNext(); )
			{
				String word = itr.next();
				System.out.println(word+"\t"+negBetas.get(word));
			}
		}
	}
	
	public static void main(String[] args) throws IOException
	{
		Parse p = new Parse();
		try
		{
			p.ClassifyMsgs("train.txt");
		} 
		catch (IOException e) 
		{
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		
		p.Statistic();
		
		//p.ShowBeta(Polarity.Irrelevant);
		System.out.println(p.GetBeta(Polarity.Irrelevant,"just","twitter"));
		System.out.println(p.GetBeta(Polarity.Neutral,"just","twitter"));
		System.out.println(p.GetBeta(Polarity.Positive,"just","twitter"));
		System.out.println(p.GetBeta(Polarity.Negative,"just","twitter"));
	}
}
